{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNPzRZhdpQoWSGn8RM7b2bg",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/LC1332/Chat-Haruhi-Suzumiya/blob/main/notebook/Erotica%E7%88%AC%E8%99%AB.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "我希望实现一个python爬虫\n",
        "\n",
        "对于这个界面\n",
        "https://book.xbookcn.net/p/all.html\n",
        "\n",
        "里面一个<div class=\"post-body entry-content\" itemprop=\"description articleBody\">的div下，提到了很多链接\n",
        "\n",
        "如《<a href=\"https://book.xbookcn.net/search/label/%E6%A2%A6%E4%B8%AD%E7%9A%84%E5%A5%B3%E5%AD%A9\" title=\"梦中的女孩\">梦中的女孩</a>》\n",
        "\n",
        "我希望对这些链接进行抽取，保存为一个list of str，方便我下一步的爬取\n"
      ],
      "metadata": {
        "id": "l_aJqDj9h-AN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y_-OONPnh4_q"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "def extract_links(url):\n",
        "    # 发送HTTP请求\n",
        "    response = requests.get(url)\n",
        "    # 检查请求是否成功\n",
        "    if response.status_code != 200:\n",
        "        print(\"请求失败，状态码：\", response.status_code)\n",
        "        return []\n",
        "\n",
        "    # 解析HTML内容\n",
        "    soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "    # 查找包含链接的div\n",
        "    div = soup.find('div', class_=\"post-body entry-content\")\n",
        "\n",
        "    # 提取所有的a标签\n",
        "    links = div.find_all('a') if div else []\n",
        "\n",
        "    # 获取所有链接的href属性\n",
        "    hrefs = [link.get('href') for link in links]\n",
        "\n",
        "    return hrefs\n",
        "\n",
        "# 目标URL\n",
        "url = \"https://book.xbookcn.net/p/all.html\"\n",
        "\n",
        "# 提取链接\n",
        "links = extract_links(url)\n",
        "\n",
        "# 打印提取的链接\n",
        "# for link in links:\n",
        "    # print(link)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(links))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n-NYHzZzkITE",
        "outputId": "72730409-7a0f-450f-a837-6dade3c526a4"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "296\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "对于links[0]\n",
        "\n",
        "是形如\n",
        "\n",
        "https://book.xbookcn.net/search/label/%E5%A4%9C%E8%89%B2%E4%B8%8B%E7%9A%84%E4%B8%8A%E6%B5%B7%E6%BB%A9\n",
        "\n",
        "的链接\n",
        "\n",
        "我希望先建立一个article_name, 把label/之后的编码转换回utf-8（主要是中文字）\n",
        "\n",
        "寻找所有的形如\n",
        "\n",
        "\n",
        "<h3 class=\"post-title entry-title\" itemprop=\"name\">\n",
        "<a href=\"https://book.xbookcn.net/2000/02/blog-post_2.html\">夜色下的上海滩 第二章</a>\n",
        "</h3>\n",
        "\n",
        "的元素，保存成一个list of dict,其中包含link和chapter_name两个字段\n"
      ],
      "metadata": {
        "id": "CJc_MX7qicRm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "from urllib.parse import unquote\n",
        "\n",
        "def extract_chapters(url):\n",
        "    # 将URL中的百分比编码转换为UTF-8编码\n",
        "    article_name = unquote(url.split('/label/')[-1])\n",
        "    # print(\"文章名称:\", article_name)\n",
        "\n",
        "    # 发送HTTP请求\n",
        "    response = requests.get(url)\n",
        "    # 检查请求是否成功\n",
        "    if response.status_code != 200:\n",
        "        print(\"请求失败，状态码：\", response.status_code)\n",
        "        return []\n",
        "\n",
        "    # 解析HTML内容\n",
        "    soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "    # 查找所有符合条件的h3标签\n",
        "    h3_tags = soup.find_all('h3', class_=\"post-title entry-title\")\n",
        "\n",
        "    # 提取链接和章节名称\n",
        "    chapters = []\n",
        "    for tag in h3_tags:\n",
        "        a_tag = tag.find('a')\n",
        "        if a_tag:\n",
        "            chapters.append({\n",
        "                'link': a_tag.get('href'),\n",
        "                'chapter_name': a_tag.get_text().strip()\n",
        "            })\n",
        "\n",
        "    return chapters, article_name\n",
        "\n",
        "# 示例链接\n",
        "chapters,article_name = extract_chapters(links[1])\n"
      ],
      "metadata": {
        "id": "mONqK9mMiAlE"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "datas = []\n",
        "\n",
        "from tqdm import tqdm\n",
        "import time\n",
        "\n",
        "for link in tqdm(links):\n",
        "    try:\n",
        "        chapters,article_name = extract_chapters(link)\n",
        "        datas.append({\n",
        "            'article_name': article_name,\n",
        "            'chapters': chapters\n",
        "        })\n",
        "        time.sleep(0.3)\n",
        "    except Exception as e:\n",
        "        print(e)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uu5GsR0kiJW0",
        "outputId": "a44cd238-bfa6-4993-f946-6d80bb1128e7"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/296 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Invalid URL '/p/popular.html': No scheme supplied. Perhaps you meant https:///p/popular.html?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 12%|█▎        | 37/296 [00:22<02:35,  1.67it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Invalid URL '/p/urban.html': No scheme supplied. Perhaps you meant https:///p/urban.html?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 32%|███▏      | 94/296 [00:59<02:21,  1.43it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Invalid URL '/p/martial.html': No scheme supplied. Perhaps you meant https:///p/martial.html?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 52%|█████▏    | 155/296 [01:39<01:29,  1.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Invalid URL '/p/fantasy.html': No scheme supplied. Perhaps you meant https:///p/fantasy.html?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 72%|███████▏  | 212/296 [02:17<00:53,  1.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Invalid URL '/p/adventure.html': No scheme supplied. Perhaps you meant https:///p/adventure.html?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 80%|████████  | 237/296 [02:33<00:38,  1.54it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Invalid URL '/p/history.html': No scheme supplied. Perhaps you meant https:///p/history.html?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 87%|████████▋ | 258/296 [02:45<00:22,  1.72it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Invalid URL '/p/dark.html': No scheme supplied. Perhaps you meant https:///p/dark.html?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 94%|█████████▍| 279/296 [02:59<00:11,  1.48it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Invalid URL '/p/romance.html': No scheme supplied. Perhaps you meant https:///p/romance.html?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 296/296 [03:10<00:00,  1.56it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "\n",
        "# datas = [1, 2, 3] # 示例数据\n",
        "\n",
        "with open('/content/link_datas.pkl', 'wb') as f:\n",
        "    pickle.dump(datas, f)"
      ],
      "metadata": {
        "id": "CSeHheu_jpEf"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i, data in  enumerate(datas):\n",
        "    print(data['article_name'],end = ' ')\n",
        "    if (i+1) % 7 == 0:\n",
        "        print()"
      ],
      "metadata": {
        "id": "Os9lQ7a7kr3n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(datas[1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fDUHKyChlgGl",
        "outputId": "d1f8ad13-6603-4c42-f6fa-6a5ce931b8ff"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'article_name': '舌战法庭', 'chapters': [{'link': 'https://book.xbookcn.net/2000/02/blog-post_786.html', 'chapter_name': '舌战法庭 内容简介'}, {'link': 'https://book.xbookcn.net/2000/02/blog-post_1.html', 'chapter_name': '舌战法庭 第一章'}, {'link': 'https://book.xbookcn.net/2000/02/blog-post_22.html', 'chapter_name': '舌战法庭 第二章'}, {'link': 'https://book.xbookcn.net/2000/02/blog-post_18.html', 'chapter_name': '舌战法庭 第三章'}, {'link': 'https://book.xbookcn.net/2000/02/blog-post_59.html', 'chapter_name': '舌战法庭 第四章'}, {'link': 'https://book.xbookcn.net/2000/02/blog-post_48.html', 'chapter_name': '舌战法庭 第五章'}, {'link': 'https://book.xbookcn.net/2000/02/blog-post_65.html', 'chapter_name': '舌战法庭 第六章'}, {'link': 'https://book.xbookcn.net/2000/02/blog-post_79.html', 'chapter_name': '舌战法庭 第七章'}]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "def extract_content(link):\n",
        "    # 发送HTTP请求\n",
        "    response = requests.get(link)\n",
        "    # 检查请求是否成功\n",
        "    if response.status_code != 200:\n",
        "        print(\"请求失败，状态码：\", response.status_code)\n",
        "        return \"\"\n",
        "\n",
        "    # 解析HTML内容\n",
        "    soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "    # 查找所有具有特定类名的div元素\n",
        "    divs = soup.find_all('div', class_=\"post-body entry-content\")\n",
        "\n",
        "    if len(divs)>1:\n",
        "        print(\"警告：找到多个div：\")\n",
        "\n",
        "    # 检查itemprop属性并选择内容最长的div\n",
        "    longest_content = \"\"\n",
        "\n",
        "    for i, div in enumerate(divs):\n",
        "\n",
        "\n",
        "        content = div.get_text()\n",
        "        if len(content) > len(longest_content):\n",
        "            longest_content = content\n",
        "\n",
        "    if divs[i].get('itemprop') != \"description articleBody\":\n",
        "            print(\"警告：找到不符合条件的div：\")\n",
        "            print(str(div)[:200])  # 打印div的前200个字符\n",
        "\n",
        "    return longest_content\n",
        "\n",
        "# 示例链接\n",
        "links = [\"https://book.xbookcn.net/2003/08/blog-post_229.html\", \"https://book.xbookcn.net/2000/02/blog-post_1.html\"]\n",
        "\n",
        "# 提取内容\n",
        "for link in links:\n",
        "    content = extract_content(link)\n",
        "    print(\"提取的内容：\", content[:200])  # 打印内容的前200个字符\n",
        "\n"
      ],
      "metadata": {
        "id": "uucpvD4Poqpb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "我已经验证 extract_content(link) 可以使用\n",
        "\n",
        "\n",
        "father_folder = \"/content/output\"\n",
        "\n",
        "for i, data in enumerate(datas):\n",
        "    chapters = data['chapters']\n",
        "    article_name = data['article_name']\n",
        "    \n",
        "    # 在father_folder下建立以article_name明明的文件夹\n",
        "\n",
        "    for j, chapter in enumerate(chapters[1:]):\n",
        "        text = extract_content(chapter['link'])\n",
        "        if text.strip() == \"\":\n",
        "            continue\n",
        "        # fname = father_folder + \"chapter_\" + str(j) + \".txt\"\n",
        "\n",
        "        # 保存text到fname中\n",
        "\n",
        "        \n",
        "我希望进一步补全下面的代码，对于每个article，建立子文件夹\n",
        "\n",
        "然后爬取text保存到对应的子文件夹中\n"
      ],
      "metadata": {
        "id": "SLc2jiAQpoHP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_dziT3YqqiZW",
        "outputId": "884dac24-6059-4b15-8a54-8db7107bdf37"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!ls /content/drive/MyDrive/Wuxia/erotica"
      ],
      "metadata": {
        "id": "KK1w_wCTq5fV"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "father_folder = \"/content/drive/MyDrive/Wuxia/erotica\"\n",
        "\n",
        "import os\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "# 假设extract_content函数已经定义\n",
        "# def extract_content(link):\n",
        "#     ...\n",
        "\n",
        "for data in tqdm(datas):\n",
        "    chapters = data['chapters']\n",
        "    article_name = data['article_name']\n",
        "\n",
        "    # 创建以article_name命名的文件夹\n",
        "    article_folder = os.path.join(father_folder, article_name)\n",
        "    if not os.path.exists(article_folder):\n",
        "        os.makedirs(article_folder)\n",
        "\n",
        "    for j, chapter in enumerate(chapters[1:]):\n",
        "        text = extract_content(chapter['link'])\n",
        "        if text.strip() == \"\":\n",
        "            continue\n",
        "\n",
        "        # 创建文件名\n",
        "        fname = os.path.join(article_folder, f\"chapter_{j}.txt\")\n",
        "\n",
        "        # 保存text到文件中\n",
        "        with open(fname, 'w', encoding='utf-8') as file:\n",
        "            file.write(text)\n",
        "\n",
        "        # print(f\"已保存：{fname}\")\n",
        "\n",
        "        time.sleep(0.1)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ElAUCF1Mlk5D",
        "outputId": "ac7bf804-de12-4a12-f91c-7c5d286bf908"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 11%|█         | 32/288 [04:59<45:25, 10.64s/it]"
          ]
        }
      ]
    }
  ]
}